from internal_scripts.data_loaders.BlackFridayDataLoader import BlackFridayDataLoader
from internal_scripts.data_loaders.LoanDataLoader import *
from internal_scripts.modelling.loading.models_loading import get_saved_models
from internal_scripts.descriptors.Eli5Descriptor import Eli5Descriptor
from internal_scripts.descriptors.ShapDescriptor import ShapDescriptor
from internal_scripts.descriptors.LimeDescriptor import LimeDescriptor
import shap
shap.initjs()
from IPython.display import display, HTML
loaders = [LoanDataLoader(), BlackFridayDataLoader()]
descriptors = [Eli5Descriptor(), ShapDescriptor(), LimeDescriptor()]
final_dict = {}
for loader in loaders:
data = loader.get_train_test_split()
data_name = data['dataset_name']
print(f"Working with {data_name}")
models = get_saved_models(data_name)
final_dict[data_name] = {}
for name, model in models.items():
print(f"Describing {name} model")
final_dict[data_name][name] = {}
for desc in descriptors:
desc_name = desc.get_descriptor_name()
print(f"Using {desc_name} descriptor")
model_desc = desc.describe(name, model, data)
final_dict[data_name][name][desc_name] = model_desc
print()
Working with Loan_Data
Using TensorFlow backend.
Describing Decision_Tree_Loan_Data model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor Decision_Tree_Loan_Data Intercept 0.4975222717425971 Prediction_local [0.72673558] Right: 1.0 Describing Logistic_Regression_Loan_Data model Using Eli 5 descriptor Using Shap descriptor Model type not yet supported by TreeExplainer: <class 'sklearn.linear_model.logistic.LogisticRegression'> Using Lime descriptor Logistic_Regression_Loan_Data Describing Random_Forest_Classifier_Loan_Data model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor Random_Forest_Classifier_Loan_Data Describing XGboost_Loan_Data model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor XGboost_Loan_Data Describing Keras_Simple_Classifier_Loan_Data model Using Eli 5 descriptor Using Shap descriptor
c:\program files\python36\lib\site-packages\shap\explainers\deep\deep_tf.py:138: UserWarning: You have provided over 5k background samples! For better performance consider using smaller random sample.
warnings.warn("You have provided over 5k background samples! For better performance consider using smaller random sample.")
0 Using Lime descriptor Keras_Simple_Classifier_Loan_Data Working with Black_Friday Describing Decision_Tree_Black_Friday model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor Decision_Tree_Black_Friday
c:\program files\python36\lib\site-packages\sklearn\utils\validation.py:595: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler. warnings.warn(msg, DataConversionWarning)
Intercept 0.3937733793596836 Prediction_local [0.284543] Right: 0.0 Describing Logistic_Regression_Black_Friday model Using Eli 5 descriptor Using Shap descriptor Model type not yet supported by TreeExplainer: <class 'sklearn.linear_model.logistic.LogisticRegression'> Using Lime descriptor Logistic_Regression_Black_Friday Describing Random_Forest_Classifier_Black_Friday model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor Random_Forest_Classifier_Black_Friday Describing XGboost_Black_Friday model Using Eli 5 descriptor Using Shap descriptor Using Lime descriptor XGboost_Black_Friday Describing Keras_Simple_Classifier_Black_Friday model Using Eli 5 descriptor Using Shap descriptor 0 Using Lime descriptor Keras_Simple_Classifier_Black_Friday
for dataset, dicts in final_dict.items():
display(HTML(f"<h1>{dataset} data</h1>"))
for model, results in dicts.items():
display(HTML(f"<h2>{model} data</h2>"))
for descriptor, results in results.items():
if len(results) == 0:
continue
display(HTML(f"<h3>{descriptor} data</h3>"))
for metric, result in results.items():
display(HTML(f"<h4>{metric} data</h4>"))
display(result)
| Weight | Feature |
|---|---|
| 0.2222 | Credit_Score |
| 0.1223 | Current_Loan_Amount |
| 0.0941 | Monthly_Debt |
| 0.0882 | Years_of_Credit_History |
| 0.0850 | Maximum_Open_Credit |
| 0.0792 | Current_Credit_Balance |
| 0.0716 | Annual_Income |
| 0.0545 | Number_of_Open_Accounts |
| 0.0454 | Months_since_last_delinquent |
| 0.0087 | Years_in_current_job_10more_years |
| 0.0072 | Years_in_current_job_6_years |
| 0.0069 | Number_of_Credit_Problems |
| 0.0067 | Bankruptcies |
| 0.0066 | Home_Ownership_Rent |
| 0.0060 | Years_in_current_job_8_years |
| 0.0060 | Years_in_current_job_2_years |
| 0.0060 | Years_in_current_job_5_years |
| 0.0059 | Years_in_current_job_3_years |
| 0.0058 | Home_Ownership_Home_Mortgage |
| 0.0058 | Years_in_current_job_less_1_year |
| … 26 more … | |
y=1 (probability 1.000) top features
| Contribution? | Feature |
|---|---|
| +0.774 | <BIAS> |
| +0.142 | Years_in_current_job_10more_years |
| +0.102 | Credit_Score |
| +0.086 | Monthly_Debt |
| +0.067 | Years_in_current_job_4_years |
| +0.058 | Years_in_current_job_less_1_year |
| +0.022 | Home_Ownership_Own_Home |
| +0.011 | Purpose_Debt_Consolidation |
| +0.005 | Number_of_Open_Accounts |
| +0.003 | Months_since_last_delinquent |
| -0.000 | Current_Loan_Amount |
| -0.015 | Annual_Income |
| -0.053 | Home_Ownership_Rent |
| -0.056 | Term_Short_Term |
| -0.143 | Maximum_Open_Credit |
y=1 top features
| Weight? | Feature |
|---|---|
| +0.000 | Monthly_Debt |
| +0.000 | Months_since_last_delinquent |
| +0.000 | Years_of_Credit_History |
| +0.000 | Annual_Income |
| +0.000 | Number_of_Open_Accounts |
| +0.000 | Current_Loan_Amount |
| +0.000 | Maximum_Open_Credit |
| +0.000 | Term_Short_Term |
| +0.000 | <BIAS> |
| +0.000 | Purpose_Debt_Consolidation |
| +0.000 | Home_Ownership_Home_Mortgage |
| +0.000 | Home_Ownership_Rent |
| +0.000 | Years_in_current_job_10more_years |
| +0.000 | Number_of_Credit_Problems |
| +0.000 | Bankruptcies |
| +0.000 | Years_in_current_job_3_years |
| +0.000 | Years_in_current_job_2_years |
| … 22 more positive … | |
| … 5 more negative … | |
| -0.000 | Term_Long_Term |
| -0.000 | Current_Credit_Balance |
| -0.000 | Credit_Score |
y=1 (probability 0.776, score 0.621) top features
| Contribution? | Feature |
|---|---|
| +0.433 | Annual_Income |
| +0.215 | Current_Loan_Amount |
| +0.171 | Monthly_Debt |
| +0.077 | Maximum_Open_Credit |
| +0.000 | Months_since_last_delinquent |
| +0.000 | Years_of_Credit_History |
| +0.000 | Number_of_Open_Accounts |
| +0.000 | <BIAS> |
| +0.000 | Purpose_Debt_Consolidation |
| +0.000 | Home_Ownership_Home_Mortgage |
| +0.000 | Years_in_current_job_7_years |
| -0.000 | Term_Long_Term |
| -0.055 | Current_Credit_Balance |
| -0.220 | Credit_Score |
| Weight | Feature |
|---|---|
| 0.2120 ± 0.0248 | Credit_Score |
| 0.1139 ± 0.0176 | Current_Loan_Amount |
| 0.0894 ± 0.0053 | Monthly_Debt |
| 0.0894 ± 0.0069 | Maximum_Open_Credit |
| 0.0877 ± 0.0084 | Current_Credit_Balance |
| 0.0835 ± 0.0073 | Years_of_Credit_History |
| 0.0748 ± 0.0086 | Annual_Income |
| 0.0629 ± 0.0067 | Number_of_Open_Accounts |
| 0.0471 ± 0.0059 | Months_since_last_delinquent |
| 0.0090 ± 0.0029 | Number_of_Credit_Problems |
| 0.0081 ± 0.0064 | Years_in_current_job_10more_years |
| 0.0078 ± 0.0023 | Bankruptcies |
| 0.0071 ± 0.0072 | Purpose_Debt_Consolidation |
| 0.0069 ± 0.0080 | Term_Long_Term |
| 0.0063 ± 0.0014 | Years_in_current_job_less_1_year |
| 0.0063 ± 0.0022 | Years_in_current_job_2_years |
| 0.0058 ± 0.0029 | Home_Ownership_Rent |
| 0.0056 ± 0.0016 | Years_in_current_job_5_years |
| 0.0054 ± 0.0013 | Years_in_current_job_1_year |
| 0.0052 ± 0.0011 | Years_in_current_job_4_years |
| … 26 more … | |
y=1 (probability 1.000) top features
| Contribution? | Feature |
|---|---|
| +0.773 | <BIAS> |
| +0.104 | Credit_Score |
| +0.104 | Months_since_last_delinquent |
| +0.037 | Current_Credit_Balance |
| +0.033 | Monthly_Debt |
| +0.021 | Years_in_current_job_7_years |
| +0.015 | Home_Ownership_Rent |
| +0.012 | Annual_Income |
| +0.008 | Years_in_current_job_2_years |
| +0.006 | Home_Ownership_Home_Mortgage |
| +0.006 | Years_in_current_job_4_years |
| +0.005 | Years_in_current_job_less_1_year |
| +0.004 | Number_of_Open_Accounts |
| +0.003 | Purpose_Home_Improvements |
| +0.002 | Home_Ownership_Own_Home |
| +0.002 | Purpose_Debt_Consolidation |
| +0.001 | Years_in_current_job_3_years |
| +0.001 | Years_in_current_job_8_years |
| +0.001 | Purpose_Other |
| +0.001 | Years_in_current_job_9_years |
| +0.001 | Tax_Liens |
| +0.001 | Years_in_current_job_-1 |
| +0.001 | Years_in_current_job_5_years |
| +0.001 | Bankruptcies |
| +0.000 | Purpose_Business_Loan |
| +0.000 | Purpose_small_business |
| +0.000 | Purpose_Buy_House |
| -0.000 | Years_in_current_job_1_year |
| -0.000 | Purpose_Medical_Bills |
| -0.000 | Purpose_Buy_a_Car |
| -0.000 | Purpose_major_purchase |
| -0.002 | Number_of_Credit_Problems |
| -0.002 | Purpose_other |
| -0.003 | Years_in_current_job_6_years |
| -0.003 | Years_in_current_job_10more_years |
| -0.004 | Current_Loan_Amount |
| -0.018 | Years_of_Credit_History |
| -0.027 | Term_Short_Term |
| -0.039 | Maximum_Open_Credit |
| -0.043 | Term_Long_Term |
| Weight | Feature |
|---|---|
| 0.3701 | Credit_Score |
| 0.1750 | Current_Loan_Amount |
| 0.1037 | Term_Long_Term |
| 0.0528 | Annual_Income |
| 0.0454 | Home_Ownership_Rent |
| 0.0409 | Home_Ownership_Home_Mortgage |
| 0.0305 | Years_in_current_job_-1 |
| 0.0210 | Current_Credit_Balance |
| 0.0196 | Monthly_Debt |
| 0.0169 | Maximum_Open_Credit |
| 0.0163 | Purpose_Business_Loan |
| 0.0160 | Purpose_small_business |
| 0.0152 | Number_of_Credit_Problems |
| 0.0144 | Years_in_current_job_3_years |
| 0.0136 | Purpose_other |
| 0.0133 | Years_of_Credit_History |
| 0.0129 | Years_in_current_job_6_years |
| 0.0107 | Number_of_Open_Accounts |
| 0.0087 | Months_since_last_delinquent |
| 0.0031 | Years_in_current_job_less_1_year |
| … 26 more … | |
y=1 (probability 0.743, score 1.063) top features
| Contribution? | Feature |
|---|---|
| +1.321 | <BIAS> |
| +0.251 | Credit_Score |
| +0.049 | Home_Ownership_Home_Mortgage |
| +0.037 | Home_Ownership_Rent |
| +0.011 | Years_in_current_job_-1 |
| +0.010 | Months_since_last_delinquent |
| +0.003 | Purpose_Business_Loan |
| +0.003 | Years_of_Credit_History |
| +0.003 | Maximum_Open_Credit |
| +0.002 | Purpose_small_business |
| +0.002 | Number_of_Credit_Problems |
| -0.015 | Current_Credit_Balance |
| -0.017 | Number_of_Open_Accounts |
| -0.021 | Annual_Income |
| -0.029 | Monthly_Debt |
| -0.235 | Current_Loan_Amount |
| -0.312 | Term_Long_Term |
| Weight | Feature |
|---|---|
| 0.3341 | Occupation |
| 0.1071 | Marital_Status |
| 0.1040 | City_Category_C |
| 0.0523 | Stay_In_Current_City_Years_2 |
| 0.0519 | Stay_In_Current_City_Years_1 |
| 0.0425 | Age_46-50 |
| 0.0380 | Stay_In_Current_City_Years_4+ |
| 0.0366 | Stay_In_Current_City_Years_3 |
| 0.0353 | Stay_In_Current_City_Years_0 |
| 0.0351 | Age_18-25 |
| 0.0258 | Age_51-55 |
| 0.0251 | Age_26-35 |
| 0.0240 | Age_36-45 |
| 0.0203 | City_Category_A |
| 0.0194 | City_Category_B |
| 0.0184 | Age_55+ |
| 0.0150 | Age_0-17 |
| 0.0148 | Gender_F |
| y=0 (probability 1.000) top features | y=1 (probability 0.000) top features | y=2 (probability 0.000) top features | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
|
| y=0 top features | y=1 top features | y=2 top features | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
|
| y=0 (probability 0.271, score -0.173) top features | y=1 (probability 0.265, score -0.195) top features | y=2 (probability 0.465, score 0.368) top features | ||||||||||||||||||||||||||||||||||||||||||||||||
|
|
|
| Weight | Feature |
|---|---|
| 0.5291 ± 0.1273 | Occupation |
| 0.0691 ± 0.0384 | Marital_Status |
| 0.0514 ± 0.0410 | Gender_F |
| 0.0488 ± 0.0459 | City_Category_C |
| 0.0354 ± 0.0366 | City_Category_B |
| 0.0268 ± 0.0127 | Stay_In_Current_City_Years_1 |
| 0.0247 ± 0.0127 | Stay_In_Current_City_Years_2 |
| 0.0244 ± 0.0161 | Stay_In_Current_City_Years_4+ |
| 0.0243 ± 0.0255 | Age_26-35 |
| 0.0239 ± 0.0169 | Stay_In_Current_City_Years_3 |
| 0.0234 ± 0.0122 | Age_18-25 |
| 0.0221 ± 0.0182 | Stay_In_Current_City_Years_0 |
| 0.0207 ± 0.0212 | Age_36-45 |
| 0.0203 ± 0.0115 | Age_51-55 |
| 0.0191 ± 0.0123 | Age_46-50 |
| 0.0164 ± 0.0081 | Age_55+ |
| 0.0107 ± 0.0113 | City_Category_A |
| 0.0094 ± 0.0038 | Age_0-17 |
| y=0 (probability 0.500) top features | y=1 (probability 0.000) top features | y=2 (probability 0.500) top features | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
|
| Weight | Feature |
|---|---|
| 0.5134 | City_Category_C |
| 0.0589 | Gender_F |
| 0.0560 | Age_55+ |
| 0.0422 | City_Category_B |
| 0.0345 | Stay_In_Current_City_Years_3 |
| 0.0309 | Age_26-35 |
| 0.0285 | Stay_In_Current_City_Years_1 |
| 0.0278 | Age_51-55 |
| 0.0268 | City_Category_A |
| 0.0249 | Age_36-45 |
| 0.0241 | Marital_Status |
| 0.0230 | Occupation |
| 0.0209 | Stay_In_Current_City_Years_0 |
| 0.0203 | Stay_In_Current_City_Years_4+ |
| 0.0180 | Age_0-17 |
| 0.0174 | Age_18-25 |
| 0.0170 | Age_46-50 |
| 0.0154 | Stay_In_Current_City_Years_2 |
| y=0 (probability 0.321, score -0.066) top features | y=1 (probability 0.263, score -0.265) top features | y=2 (probability 0.417, score 0.195) top features | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
|